'''
Parse online table
'''
import urllib.request, os, time, shutil,  sys
from html.parser import HTMLParser
from common import Path

class HTMLParser_BBC(HTMLParser):
    def __init__(self):
        self.data_list = []
        
        self.reset()
        self.tag = 0
          
    def handle_starttag(self, tag, attr):
        if tag in ['td',  'a', 'span', 'b']:
            #print ('tag', attr)
            self.tag = 1
        else:
            self.tag = 0

    def handle_data(self, data):
        if self.tag:
            if data.isspace() or data in ['Last', '[R]']:
                return
            else:
                self.data_list.append(data)
                #print (data)

class HTMLParser_LFC_HISTORY(HTMLParser):
    def __init__(self):
        self.data_list = []
        
        self.reset()
        self.tag = 0

    def handle_starttag(self, tag, attr):
#        print ('s', tag, attr)
        if tag in ['td' 'a']:
            self.tag = 1
        else:
            self.tag = 0
                                 
    
    def handle_data(self, data):
        #print (data)
        if self.tag >= 0:
            if not data.isspace():
                self.data_list.append(data)

              
class SourceDownloader():
    def __init__(self, url = 'http://www.bbc.co.uk/sport2/hi/football/eng_prem/table'):
        try:
            print ('Opening ', url)
            self.html = urllib.request.urlopen(url).read().decode('iso-8859-1')
        except:
            print ("no data for url ", url)
            self.html = NULL
        


class ContentProcessor():
    def __init__(self, data, parser = 'bbc'):

        if parser == 'bbc':
            self.start_tag  = "<tbody>"
            self.end_tag    = "<tbody>"            
            self.parser = HTMLParser_BBC()
            self.parser.strict = True
            self.markup_rules_call = self.markup_rules
            self.opp_id_rules_call = self.opp_id_rules
            self.opp_rus_name_rules_call = self.opp_rus_name_rules
            
        elif parser == 'lfch':
            self.start_tag  = '<tbody>'
            self.end_tag    = '</tbody>'
            self.parser = HTMLParser_LFC_HISTORY()
            self.parser.strict = True
            self.opposition_data = self.import_sql('data/opposition.sql', [1, 2, 4, 5])         
            self.markup_rules_call = self.lfc_history_rules_markup
            self.opp_id_rules_call = self.lfc_history_rules_id
            self.opp_rus_name_rules_call = self.lfc_history_rules_name
            
        elif parser == 'lfco':
            self.start_tag  = "tbody"
            self.end_tag    = "/tbody"            
            self.parser = HTMLParser_BBC()
            self.parser.strict = True
            self.markup_rules_call = self.markup_rules
            self.opp_id_rules_call = self.opp_id_rules
            self.opp_rus_name_rules_call = self.opp_rus_name_rules
        else:
            assert False, 'unknown parser'
                      
        start = data.find(self.start_tag)
        assert start != -1, 'No start tag <{0}>'.format(self.start_tag)
            
        end = data.find(self.end_tag)
        assert end != -1, 'No end tag <{0}>'.format(self.end_tag)

        try:
            #print(data[start:end])
            self.parser.feed(data[start:end])
        except:
            print ("Exception!")
            
        self.filter_data_list()
            
    def markup_rules(self, table_line):
        
        pos_index = ['first', 'cl-football', 'cl-football', 'cl-football', 'uefa', 'uefa',\
                     '','','','','','','','','','','',\
                     'relegation','relegation','relegation']
        
        p1 = pos_index[int(table_line[0])-1]
        p2 = 'liverpool-stats' if table_line[1] == 'Liverpool' else ''
        return p2 + ' ' + p1
        
    def lfc_history_rules_markup(self, table_line):
        p1 = 'first' if table_line[0] == 1 else ''
        p2 = 'liverpool-stats' if table_line[1] == 'Liverpool FC' else ''
        return p2 + ' ' + p1        
        
        
    def opp_id_rules(self, name):
        
        """ The following table encodes PL teams in liverbird.ru database, please note Liverpool id equals zero """
        clubs_list = {'Burnley': '1431', 'Chelsea': '35', 'Tottenham Hotspur': '149', 'Manchester United': '39', 'Manchester City': '38', 'Stoke City': '47', 'Arsenal': '31', 'Liverpool': '0',\
                      'Aston Villa': '32', 'Sunderland':'48', 'West Ham United': '42', 'Birmingham': '1433', 'Wolverhampton': '1435', 'Hull City':'37', 'Fulham': '36',\
                      'Everton': '27', 'Wigan': '46', 'Blackburn': '33', 'Bolton': '34', 'Portsmouth': '43', 
                      'Newcastle United': '41', 'Norwich City': '5128', 'QPR': '5144', 'Swansea City': '5171', 'West Bromwich Albion': '59', 'West Ham United': '42', 'Reading':'2832', 'Southampton':'5161',
                      'Crystal Palace': '5059', 'Cardiff City': '5048', 'Leicester City': '5110', 'Queens Park Rangers': '5144', 'Burnley': '1431'}

        return clubs_list[name]

    def opp_rus_name_rules(self, eng_name, short = False):
        rus_name = {'Chelsea': 'Челси', 'Tottenham Hotspur': 'Тоттенхэм Хотспур', 'Manchester United': 'Манчестер Юнайтед', 'Manchester City': 'Манчестер Сити',\
                    'Arsenal': 'Арсенал', 'Liverpool': 'Ливерпуль', 'Aston Villa': 'Астон Вилла', 'Sunderland':'Сандерленд',\
                    'Wolverhampton': 'Вулверхэмптон Уондерерс', 'Fulham': 'Фулхэм', 'Everton': 'Эвертон', 'Wigan': 'Уиган Атлетик',\
                    'Bolton': 'Болтон Уондерерс', 'Newcastle United': 'Ньюкасл Юнайтед', 'Norwich City': 'Норвич Сити', 'QPR': 'Куинс Парк Рейнджерс',\
                    'Swansea City': 'Суонси Сити', 'West Bromwich Albion': 'Вест Бромвич Альбион', 'Blackburn': 'Блэкберн Роверс', 'Stoke City': 'Стоук Сити',
                    'West Ham United': 'Вест Хэм Юнайтед', 'Reading': 'Рединг', 'Southampton': 'Саутгемптон', \
                    'Crystal Palace': 'Кристал Пэлас', 'Cardiff City': 'Кардифф Сити', 'Hull City':'Халл Сити', 'Leicester City': 'Лестер Сити', 'Queens Park Rangers': 'Куинс Парк Рейнджерс', 'Burnley': 'Бёрнли'}


        rus_name = {'Chelsea': 'Челси', 'Tottenham Hotspur': 'Тоттенхэм', 'Manchester United': 'Ман. Юнайтед', 'Manchester City': 'Ман. Сити',\
                    'Arsenal': 'Арсенал', 'Liverpool': 'Ливерпуль', 'Aston Villa': 'Астон Вилла', 'Sunderland':'Сандерленд',\
                    'Wolverhampton': 'Вулверхэмптон', 'Fulham': 'Фулхэм', 'Everton': 'Эвертон', 'Wigan': 'Уиган',\
                    'Bolton': 'Болтон', 'Newcastle United': 'Ньюкасл', 'Norwich City': 'Норвич', 'QPR': 'КПР',\
                    'Swansea City': 'Суонси', 'West Bromwich Albion': 'ВБА', 'Blackburn': 'Блэкберн', 'Stoke City': 'Стоук',
                    'West Ham United': 'ВХЮ', 'Reading': 'Рединг', 'Southampton': 'Саутгемптон', \
                    'Crystal Palace': 'Кристал Пэлас', 'Cardiff City': 'Кардифф', 'Hull City':'Халл', 'Leicester City': 'Лестер', 'Queens Park Rangers': 'КПР', 'Burnley': 'Бёрнли'}

        if short:
            return rus_name_short[eng_name]
        else:
            return rus_name[eng_name]

    def lfc_history_rules(self, name):
        exceptions = [[0,     'Ливерпуль',                  'Liverpool FC'],
                      [46,    'Уиган',                      'Wigan Ath.'],
                      [1435,  'Вулверхэмптон Уондерерс',    'Wolves'],
                      [5155,  'Шеффилд Юнайтед',            'Sheff Utd'],
                      [5156,  'Шеффилд Уэнсдей',            'Sheff Wed'],                                          
                      [5129,  'Ноттингем Форест',           'Nottm For'],
                      [5144,  'Куинс Парк Рейнджерс',       'QPR'],                                          
                      [5037,  'Брэдфорд Парк Авеню',        'Bradford Park Av.'],
                      [5156,  'Шеффилд Уэнсдей',            'The Wednesday'],                                          
                      [31,    'Арсенал',                    'Woolwich Arsenal'],
                      [1433,  'Бирмингем Сити',             'Small Heath'],       
                      [5083,  'Гейнсборо Тринити',          'Gainsborough Tr.'],
                      [5110,  'Лестер Сити',                'Leicester Fosse'],                                          
                      [5143,  'Порт Вейл',                  'Burslem Port Vale'],
                      [5180,  'Уолсолл',                    'Walsall Town Swifts'],                  
                      [40,    'Мидлсбро',                   'Middlesbrough I.'],
                      [5127,  'Нортвич Виктория',           'Northwich Vict.'],              
                      [39,    'Манчестер Юнайтед',          'Newton Heath'],
                      [5171,  'Суонси Сити',                'Swansea City'],
                      
                      ]
        
        for item in exceptions:
            if name in item:
                break
        else:
            for item in self.opposition_data:
                if name in item:
                    break
                if item[2].find(name) >= 0:
                    break
            else:
                print ('---- Not found id for', name)
        
        
        return item        

    def lfc_history_rules_id(self, name):
        return self.lfc_history_rules(name)[0]

    def lfc_history_rules_name(self, name):
        return self.lfc_history_rules(name)[1]

    def small_table_range(self, lfc_position):
        width = 2
        ul = lfc_position - width
        ll = lfc_position + width

        if ul <= 0:
            ''' Upper table '''
            ul = 1
            ll = 5
        elif ll >= 20:
            ''' Lower table '''
            ul = 16
            ll = 20

        self.upper_place = ul
        self.lower_place = ll


    def filter_data_list(self):
        clubs_list_lfco = ['Chelsea', 'Tottenham Hotspur', 'Manchester United', 'Manchester City', 'Stoke City', 'Arsenal', 'Liverpool',
                      'Aston Villa', 'Sunderland', 'West Ham United', 'Burnley', 'Everton', 'Leicester City', 
                      'Newcastle United', 'Queens Park Rangers', 'Hull City', 'Swansea City', 'West Bromwich Albion', 'Crystal Palace', 'Southampton']

        self.filtered_dict = {}

        for club in clubs_list_lfco:
            pos = self.parser.data_list.index(club)

            if club == 'Liverpool':
                self.small_table_range(int(self.parser.data_list[pos-1]))

            self.filtered_dict[int(self.parser.data_list[pos-1])] = self.parser.data_list[pos:pos+19]


    def generate_inserts(self, db_mode = False):
        gmtime = time.gmtime()

        commands = []        
        for c in self.filtered_dict.values():
            line =\
                [date, position, club_id, games, won_h, drawn_h, lost_h, scored_h, conceded_h, pts_h,won_a, drawn_a, lost_a, scored_a, conceded_a, pts_a, won, drawn, lost, scored, conceded, pts] =\
                ['{0}{1:02}{2:02}'.format(gmtime.tm_year,gmtime.tm_mon,gmtime.tm_mday), c[0], self.opp_id_rules_call(c[1]), c[2], c[3], c[4], c[5],  c[6], c[7], int(c[3])* 3 + int(c[4]),c[8], c[9], c[10], c[11], c[12],\
                int(c[8])* 3 + int(c[9]), int(c[3]) + int(c[8]), int(c[4]) + int(c[9]), int(c[5]) + int(c[10]), int(c[6]) + int(c[11]), int(c[7]) + int(c[12]), c[14]]
            
            commands.append('INSERT INTO league_table VALUES (' +  ''.join('\'{}\','.format(item) for item in line)[:-1] + ');\n')
             
        if db_mode:
            from db import DB_Agent
            db = DB_Agent()
            db.connect()
            db.execute('SELECT COUNT(*) FROM league_table;')
            for line in commands:
                db.execute(line, silent = True)
            db.execute('SELECT COUNT(*) FROM league_table;')
            db.close()
        else:
            with open('table.sql', 'w') as out_file:
                for line in commands:
                    out_file.write(line)

    def generate_html(self, dest_file, upper_place = 1, lower_place = 20, short = False):
    
        destination = open(dest_file, 'w', encoding = 'utf-8')
        print('Writing ', dest_file)

        if short:
            destination.write('<h2>Премьер-лига</h2><table class=\"short-league-table\"><tbody>')
        else:
            destination.write('<table id=\"epl-table\" class=\"league-table\">\n\
                            <thead>\n\
                            <tr>\n\
                            <td colspan=\"3\" class=\"club-info-column stat-column\"></td>\n\
                            <td colspan=\"8\" class=\"home-stat-column stat-column\">Дома</td>\n\
                            <td colspan=\"8\" class=\"away-stat-column stat-column\">В гостях</td>\n\
                            <td colspan=\"5\" class=\"total-stat-column stat-column\">Итого</td>\n\
                            <td colspan=\"2\" class=\"summary-stat-column stat-column\"></td>\n\
                            </tr>\n\
                            <tr>\n\
                            <th class=\"club-info\">#</th>\n\
                            <th class=\"club-info\">Клуб</th>\n\
                            <th class=\"total-stat-summary summary-stat stat-close\">Pl</th>\n\
                            <th class=\"home-stat-summary summary-stat stat-close\">Pl</th>\n\
                            <th class=\"home-stat stat\">W</th>\n\
                            <th class=\"home-stat stat\">D</th>\n\
                            <th class=\"home-stat stat\">L</th>\n\
                            <th class=\"home-stat stat\">+</th>\n\
                            <th class=\"home-stat stat stat-close\">-</th>\n\
                            <th class=\"home-stat-summary summary-stat stat-close\">+/-</th>\n\
                            <th class=\"home-stat-summary summary-stat stat-close\">Pts</th>\n\
                            <th class=\"away-stat-summary summary-stat stat-close\">Pl</th>\n\
                            <th class=\"away-stat stat\">W</th>\n\
                            <th class=\"away-stat stat\">D</th>\n\
                            <th class=\"away-stat stat\">L</th>\n\
                            <th class=\"away-stat stat\">+</th>\n\
                            <th class=\"away-stat stat stat-close\">-</th>\n\
                            <th class=\"away-stat-summary summary-stat stat-close\">+/-</th>\n\
                            <th class=\"away-stat-summary summary-stat stat-close\">Pts</th>\n\
                            <th class=\"total-stat stat\">W</th>\n\
                            <th class=\"total-stat stat\">D</th>\n\
                            <th class=\"total-stat stat\">L</th>\n\
                            <th class=\"total-stat stat\">+</th>\n\
                            <th class=\"total-stat stat stat-close\">-</th>\n\
                            <th class=\"total-stat-summary summary-stat\">+/-</th>\n\
                            <th class=\"total-stat-summary summary-stat\">Pts</th>\n\
                            </tr>\n\
                            </thead>\n\
                            <tbody>\n')
        
        for i in range(upper_place,lower_place + 1):
            
            c = self.filtered_dict[i]
            if not c[0] == i:
                c.insert(0, i)
   
            first_tag = self.markup_rules_call(c)
            club_id = self.opp_id_rules_call(c[1])
            if club_id == '0':
                club_name = '<a href=\"http://liverbird.ru/">{0}</a>'.format(self.opp_rus_name_rules_call(c[1]))
            else:
                club_name = '<a href=\"http://liverbird.ru/opposition/{0}">{1}</a>'.format(club_id, self.opp_rus_name_rules_call(c[1]))

            if short:
                out_str = '<tr class=\"{0}\">\n\
                        <td class=\"club-info\">{1}.</td>\n\
                        <td class=\"club-info\">{2}</td>\n\
                        <td class=\"total-stat-summary summary-stat\">{3}</td>\n\
                        </tr>\n'.format(first_tag, c[0], club_name,   c[-2])

            else:
                out_str = '<tr class=\"{0}\">  \n\
                        <td class=\"club-info\">{1}</td>\n\
                        <td class=\"club-info\">{3}</td>\n\
                        <td class=\"total-stat-summary summary-stat stat-close\">{4}</td>\n\
                        <td class=\"home-stat-summary summary-stat stat-close\">{22}</td>\n\
                        <td class=\"home-stat stat\">{5}</td>\n\
                        <td class=\"home-stat stat\">{6}</td>\n\
                        <td class=\"home-stat stat\">{7}</td>\n\
                        <td class=\"home-stat stat\">{8}</td>\n\
                        <td class=\"home-stat stat stat-close\">{9}</td>\n\
                        <td class=\"home-stat-summary summary-stat stat-close\">{24}</td>\n\
                        <td class=\"home-stat-summary summary-stat stat-close\">{25}</td>\n\
                        <td class=\"away-stat-summary summary-stat stat-close\">{23}</td>\n\
                        <td class=\"away-stat stat\">{10}</td>\n\
                        <td class=\"away-stat stat\">{11}</td>\n\
                        <td class=\"away-stat stat\">{12}</td>\n\
                        <td class=\"away-stat stat\">{13}</td>\n\
                        <td class=\"away-stat stat stat-close\">{14}</td>\n\
                        <td class=\"away-stat-summary summary-stat stat-close\">{26}</td>\n\
                        <td class=\"away-stat-summary summary-stat stat-close\">{27}</td>\n\
                        <td class=\"total-stat stat\">{15}</td>\n\
                        <td class=\"total-stat stat\">{16}</td>\n\
                        <td class=\"total-stat stat\">{17}</td>\n\
                        <td class=\"total-stat stat\">{18}</td>\n\
                        <td class=\"total-stat stat\">{19}</td>\n\
                        <td class=\"total-stat-summary summary-stat\">{20}</td>\n\
                        <td class=\"total-stat-summary summary-stat\">{21}</td>\n\
                        </tr>\n'.format(first_tag, c[0], 'NULL', club_name, \
                                        c[2], c[3], c[4], c[5], c[6], c[7],c[8], c[9], c[10], c[11], c[12],\
                                        int(c[3]) + int(c[8]), int(c[4]) + int(c[9]), int(c[5]) + int(c[10]), \
                                        int(c[6]) + int(c[11]), int(c[7])+ int(c[12]), c[-1], c[-2],\
                                        int(c[3]) + int(c[4]) + int(c[5]), int(c[8]) + int(c[9])+ int(c[10]),\
                                        int(c[6]) - int(c[7]),  int(c[3])* 3 + int(c[4]), \
                                        int(c[11])- int(c[12]), \
                                        int(c[8])* 3 + int(c[9])\
                                        )


            ''' total wins, draws, losses, goals scored and conceded, goal difference and points '''
            ''' played home and away, goal difference and points home and away'''
                
            destination.write(out_str)
 
        if short:
            destination.write('</tbody> \n </table> <small> <a href="http://www.liverbird.ru/league_table">Вся таблица</a> </small> <br>')
        else:
            timestamp = "<small>Обновлено {0.tm_year}/{0.tm_mon}/{0.tm_mon} в {0.tm_hour:02}:{0.tm_min:02} по Гринвичу</small>".format(time.gmtime())
 
            destination.write('</tbody> \n </table> ' + timestamp)

        destination.close()


    def manage_backup(self, local):

        if local:
            return

        gmtime = time.gmtime()
        todays_path =  path.archive_path +  "{}_{}_{}.html".format(gmtime.tm_year,gmtime.tm_mon,gmtime.tm_mday)

        if os.path.isfile(todays_path):
            print ('Already archived')
        else:
            print ('Archive not found, writing ', todays_path)
            shutil.copy(path.full_pl_table, todays_path)


def process_lfc(local):
    
    if  local:
        resp = SourceDownloader(local, path = 'data/table1112.htm')
        content = ContentProcessor(resp.html, bbc = False)
        content.generate_html(local, 'table1112_done.html')
    else:
        for season in range (121, 122):
            resp = SourceDownloader(False, url = 'http://www.lfchistory.net/SeasonArchive/LeagueTable/{0}'.format(season))
            content = ContentProcessor(resp.html, False)
            content.generate_html(local, 'out2/table{0}.html'.format(season))

def generate_filenames():
    for season in range (1, 120):
        url = 'http://www.lfchistory.net/SeasonArchive/LeagueTable/{0}'.format(season)
        data = urllib.request.urlopen(url).read().decode('iso-8859-1')
        x = data.find('The league table for the')
        str_season = data[x+25:x+34]
        
        command = 'mv table{0}.html table{1}.html'.format(season, str_season)
        print (command)



def process_bbc(local):
    print ('Starting. Local = {0}'.format(local))

    content = ContentProcessor(SourceDownloader(local).html)
    content.generate_html(local)
    content.manage_backup(local)


def process_pl(local):
    source = SourceDownloader(url = 'http://www.lfconline.com/league_tables/2014_2015/liverpool/index.shtml').html
    content = ContentProcessor(source, parser = 'lfco')
    content.generate_html(dest_file = path.short_table, upper_place =  content.upper_place, lower_place = content.lower_place, short = True)
    content.generate_html(dest_file = path.full_pl_table)

    if not local:
        content.manage_backup(local)

def rerun_old_tables(local):    
    raw_dict = {}
    
    res_file = open('data/table_inserts.txt', 'w') 
    
    
    with open('data/a22', 'r') as source:
        for line in source.readlines():
            [raw_date, value] = line.strip().split(':')
            d = raw_date.split('.')[0].split('_')
            date = '{}{:02}{:02}'.format(int(d[0]), int(d[1]), int(d[2]))
            value = int(value[1:])
             
            if date in raw_dict:
                raw_dict[date].append(value)
            else:
                raw_dict[date] = [value]
        
    last_sum = 0        
    
    for date in sorted(raw_dict.keys()):
        
        if len(raw_dict[date]) != 520:
            print (date, 'skipped, len ',len(raw_dict[date]))
            continue

        check = sum(raw_dict[date])
        if check == last_sum:
            print (date, 'skipped - same data')
            continue
        else:
            last_sum = check
                
        print (date, check)
        
        for i in range(0, 20):
            c = raw_dict[date][i*26:i*26+25]

            line =\
                    [game_date, position, club_id, games,\
                      won_h, drawn_h, lost_h, scored_h, conceded_h, pts_h,\
                      won_a, drawn_a, lost_a, scored_a, conceded_a, pts_a, \
                      won, drawn, lost, scored, conceded, pts] =\
                    [date, c[0], c[1], c[2],\
                     c[4], c[5],  c[6], c[7], c[8], int(c[4])* 3 + int(c[5]),\
                     c[12], c[13], c[14], c[15], c[16], int(c[12])* 3 + int(c[13]),\
                     int(c[4]) + int(c[12]), int(c[5]) + int(c[13]), int(c[6]) + int(c[14]), int(c[7]) + int(c[15]), int(c[8]) + int(c[16]), int(c[4])* 3 + int(c[5]) + int(c[12])* 3 + int(c[13])]
             
            res_file.write ('INSERT INTO league_table VALUES (' +  ''.join('\'{}\','.format(item) for item in line)[:-1] + ');\n')

    res_file.close()
 

''' Main '''
if len(sys.argv) == 2 and sys.argv[1] == 'local':
    local = True
else:
    local = False

path = Path(local)
process_pl(local)

